#encoding=utf-8
import threading
import re
#from Numeric import *
from math import *
import shelve

#这是预处理数据，让他们转换成列表来处理
def preprocess():
	infile = open('trainSetTemp.txt')
	outfile = open('predata','w')
	lst = []
	count = 0
	last_per_id = 0
	last_per_id = 0
	sum_score = 0
	person_id = 0
	movie_id = 0
	movie_score = 0.0
	Total  = 0
	for line in infile.readlines():
		each = line.split('::')
		try:
			person_id = int( each[0] )
			movie_id = int ( each[1] )
			movie_score = float( each[4][0:-2] )
		except:
			continue	
		if ( last_per_id == 0 or last_per_id != person_id):
			if last_per_id != 0:
				lst.append({'num':count})
			if lst:
				outfile.write(str(lst)+'\n')
			lst = []
			last_per_id = person_id
			last_movie_id = movie_id
			lst.append({'user_id':person_id})
			lst.append([{'movie':movie_id,'score':movie_score}])
			count = 1
			sum_score = movie_score
			Total += 1
			continue
		if (last_per_id == person_id and last_movie_id != movie_id):
			lst[1].append({'movie':movie_id,'score':movie_score})
			last_movie_id = movie_id
			count += 1	
			sum_score += movie_score
	infile.close()
	outfile.close()
	print Total

#pearson算法计算线性相似	
def Pearson(temp_lst1,temp_lst2):
	n = len(temp_lst1)
	a=zeros(n,Float)    #creating an empty array
	b=zeros(n,Float)    #creating an empty array
	sum_x=0
	sum_y=0
	sum_XX=0
	sum_YY=0
	sum_XY=0
	for i in range(n):
		x = temp_lst1[i]['score']
		y = temp_lst2[i]['score']
		a[i]=x
		b[i]=y
		sum_x=sum_x+x
		sum_y=sum_y+y
	p=sum_x/n    #mean of x
	q=sum_y/n    #mean of y
	for i in range(n):
		X=a[i]-p
		Y=b[i]-q
		XX=pow(X,2)
		YY=pow(Y,2)
		sum_XX=sum_XX+XX
		sum_YY=sum_YY+YY
		XY=X*Y
		sum_XY=sum_XY+XY
	try:
		r=sum_XY/(sqrt(sum_XX*sum_YY))
		#r= r * pow( abs(r), 1.5)
		return r
	except:
		return -2

#Cosine Similarity
def cosine(temp_lst1,temp_lst2):
	n = len(temp_lst1)
	a=zeros(n,Float)    #creating an empty array
	b=zeros(n,Float)    #creating an empty array
	up_sum = 0.0
	down_sum = 0.0
	for i in range(n):
		x = temp_lst1[i]['score']
		y = temp_lst2[i]['score']
		up_sum += x * y
		down_sum += sqrt( pow( x, 2) ) * sqrt( pow( y, 2) )
	sim = up_sum / down_sum
	return sim

	



		
def my_sort_fun(a,b):
	if a['sim'] < b['sim']:
		return 1
	else :
		return -1	
		
def similar():
	db = shelve.open( 'neighbour.db', writeback = True )
	
	temp_file = open('predata')
	whole_dict = dict()
	for line1 in temp_file.readlines():
		temp = eval(line1)	
		temp_id = temp[0]['user_id']
		whole_dict[temp_id] = dict()
	temp_file.close()
	infile1 = open('predata')
	lst1 = []
	lst2 = []
	x = 1
	for line1 in infile1.readlines():
		lst1 = eval(line1)
		infile2 = open('predata')
		#sim_lst = []
		#sim_lst.append({'host':lst1[0]['user_id']})
		host_id = lst1[0]['user_id']
		count = 0
		for line2 in infile2.readlines()[x:]:
			lst2 = eval(line2)
			temp_lst1 = []
			temp_lst2 = []
			for each1 in lst1[1]:
				movie_id_1 = each1['movie']
				for each2 in lst2[1]:
					movie_id_2 = each2['movie']
					if (movie_id_1 == movie_id_2):
						temp_lst1.append(each1)
						temp_lst2.append(each2)
			if (temp_lst1 != [] and temp_lst2!= []):		
				#sim = Pearson(temp_lst1,temp_lst2)
				sim = cosine(temp_lst1,temp_lst2)
				if abs( sim ) > 0.3:					
					neighbor_id = lst2[0]['user_id']
					whole_dict[host_id][neighbor_id] = sim
					whole_dict[neighbor_id][host_id] = sim
					count += 1

		
		db[ str( host_id ) ] = sorted(whole_dict[host_id].iteritems(), key=lambda d:d[1], reverse = True )
		x += 1
	infile1.close()
	infile2.close()
	db.close()

if __name__=="__main__":
	preprocess()
	#similar()
